﻿using JustMathCore;
using JustMathCore.FilePool;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

namespace JMCMD
{
    public class ImportAddr
    {
        //public static void DoImport(string type, string[] srcfiles, string outdir)
        //{
        //    var txtdat = System.Text.Encoding.UTF8.GetBytes(type);
        //    if (txtdat.Length > 8) txtdat = txtdat.Take(8).ToArray();
        //    //开启写线程
        //    LinePool curpool = new LinePool(outdir, 32);
        //    BitArray bitarray = BitArray.Create();
        //    ////开8个线程写
        //    //curpool = new FilePool.LinePool(type + "_data", 32);

        //    ThreadWriteMgr.Start(curpool, 16);

        //    string[] src = new string[8];
        //    foreach (var f in srcfiles)
        //    {

        //        DateTime t0 = DateTime.Now;
        //        var finfo = new System.IO.FileInfo(f);
        //        Console.WriteLine("begin " + f + " at:" + t0.ToLongTimeString());

        //        if (finfo.Length > 1 * 1024 * 1024 * 1024)
        //        {
        //            Console.WriteLine("file is too large,use stream mode.");
        //            using (var fs = finfo.OpenRead())
        //            {
        //                using (var sr = new System.IO.StreamReader(fs))
        //                {
        //                    string[] lines = new string[16 * 1024 * 1024];
        //                    int linecount = 0;
        //                    while (true)
        //                    {
        //                        var line = sr.ReadLine();
        //                        if (line == null)
        //                            break;
        //                        lines[linecount] = line;
        //                        linecount++;
        //                        if (linecount == lines.Length)
        //                        {
        //                            Console.WriteLine("stream mode step.");

        //                            PForLines(txtdat, bitarray, lines, lines.Length);
        //                            linecount = 0;
        //                        }
        //                    }
        //                    if (linecount > 0)
        //                    {
        //                        PForLines(txtdat, bitarray, lines, linecount);
        //                    }
        //                    lines = null;
        //                }
        //            }
        //        }
        //        else
        //        {
        //            var lines = System.IO.File.ReadAllLines(f);
        //            PForLines(txtdat, bitarray, lines, lines.Length);
        //        }

        //        var time = (DateTime.Now - t0).TotalSeconds;
        //        Console.WriteLine("=>end " + f + " " + time);
        //        ThreadWriteMgr.WaitWrite();
        //    }
        //    System.IO.File.WriteAllBytes(System.IO.Path.Combine(outdir, "bloom4G.fbin"), bitarray.Memory);
        //    //);
        //    ThreadWriteMgr.WaitToExit();

        //    curpool.Dispose();
        //}

        //private static void PForLines(byte[] txtdat, BitArray bitarray, string[] lines, int count)
        //{

        //    System.Threading.Tasks.Parallel.For(0, 8, (seed) =>
        //    {
        //        for (var l = seed; l < count; l += 8)
        //        {
        //            var line = lines[l];
        //            {
        //                var data = new byte[32];
        //                if (line.Length == 40)
        //                {
        //                    for (var i = 0; i < 20; i++)
        //                    {
        //                        var d = byte.Parse(line.Substring(i * 2, 2), System.Globalization.NumberStyles.HexNumber);
        //                        data[i] = d;
        //                    }
        //                }
        //                else if (line.Length == 34)
        //                {
        //                    var b58 = AddrTool.base58Decode(line);
        //                    for (var i = 0; i < 20; i++)
        //                    {
        //                        data[i] = b58[i + 1];
        //                    }
        //                }
        //                else
        //                {
        //                    //如果遇到不规范的忽略他，只支持两种文本导入格式
        //                }

        //                for (var i = 0; i < txtdat.Length; i++)
        //                {
        //                    data[24 + i] = txtdat[i];
        //                }
        //                var index = BitConverter.ToUInt32(data, 0);
        //                lock (bitarray)
        //                {
        //                    bitarray[index] = true;
        //                }
        //                ThreadWriteMgr.Queue(data);
        //            }
        //        }
        //    }
        //      );
        //}

        public static void DoInsertAddr2Db(string dbpath, string[] srcfiles)
        {
            JustMathCore.RocksDB.DB db = new JustMathCore.RocksDB.DB();
            db.Open(dbpath, true);
            var wb = db.CreateWriteBatch();
            foreach (var f in srcfiles)
            {
                var bytes = System.IO.File.ReadAllBytes(f);
                uint max = BitConverter.ToUInt32(bytes, 0);
                uint cur = BitConverter.ToUInt32(bytes, 4);
                for (var i = 0; i < cur; i++)
                {
                    int index = 8 + i * 32;
                    byte[] key = new byte[20];
                    //20 bytes for key
                    for (var bi = 0; bi < 20; bi++)
                    {
                        key[bi] = bytes[index + bi];
                    }
                    byte[] value = new byte[8];
                    for (var bi = 0; bi < 8; bi++)
                    {
                        value[bi] = bytes[index + 24 + bi];
                    }
                    wb.Put(key, value);
                    if (wb.Count() > 1000000)
                    {
                        db.Write(wb);
                        wb.Clear();
                    }
                }
                Console.WriteLine("finish " + f + " " + DateTime.Now.ToLongTimeString());

            }
            if (wb.Count() > 1000000)
            {
                db.Write(wb);
                wb.Clear();
            }
            wb.Dispose();
            db.Dispose();

        }

        public static void DoTestAddr_Db(string dbpath, JustMathCore.AddrTools.AddrType type, string[] addrs)
        {
            Console.WriteLine("init bloom filter.");
            BitArray160 bt160 = BitArray160.Load(dbpath);
            Console.WriteLine("init bloom filter ok.");
            //JustMathCore.RocksDB.DB db = new JustMathCore.RocksDB.DB();
            //db.Open(dbpath, false);
            var logfile = System.IO.Path.Combine(dbpath, "log_" + DateTime.Now.ToFileTimeUtc() + ".txt");
            string tag = "";
            var conv = AddrTools.GetConv(type);
            foreach (var add in addrs)
            {
                JustMathCore.Hex160 hex = conv.Addr2Hash(add);
                //if (add.Length == 40 || add.Length == 42)
                //{
                //    hex = JustMathCore.Hex160.FromString(add);
                //}
                //else if (add.Length == 34)
                //{
                //    hex = Hex160.FromBytes(AddrTools.base58Decode(add).Skip(1).Take(20).ToArray());
                //}
                //else if (add.Length == 43)
                //{
                //    var addr = add.Substring(9);
                //    hex = Hex160.FromBytes(AddrTools.base58Decode(addr).Skip(1).Take(20).ToArray());
                //    tag = add.Substring(0, 8);
                //}
                //else
                //{
                //    throw new Exception("error format.");
                //}

                var result = bt160.Exist(hex);
                if (result == true)
                {
                    var line = "find:" + hex.Format() + " " + add + " tag=" + tag + "\n";
                    Console.WriteLine(line);
                    System.IO.File.AppendAllText(logfile, line);
                }
            }
            Console.WriteLine("done");
            //db.Dispose();

        }
        public static void DoTestLogFiles(string dbpath, string[] files)
        {
            JustMathCore.RocksDB.DB db = new JustMathCore.RocksDB.DB();
            db.Open(dbpath, false);
            Console.WriteLine("db open.");
            var logfile = System.IO.Path.Combine(dbpath, "finallog_" + DateTime.Now.ToFileTimeUtc() + ".txt");
            string tag = "";

            foreach (var f in files)
            {
                string[] addrs = System.IO.File.ReadAllLines(f);
                foreach (var add in addrs)
                {
                    var addr = add.Substring(5, 40);
                    JustMathCore.Hex160 hex = Hex160.FromString(addr);

                    var result = db.Get(hex) != null;
                    if (result == true)
                    {
                        var line = "find:" + hex.Format() + " " + add + " tag=" + tag + "\n";
                        Console.WriteLine(line);
                        System.IO.File.AppendAllText(logfile, line);

                    }
                }
                Console.WriteLine("done " + f + DateTime.Now.ToLongTimeString());

            }

        }
        public static void DoTestOldAddrFiles(string dbpath, string[] files)
        {
            Console.WriteLine("init bloom filter.");
            BitArray160 bt160 = BitArray160.Load(dbpath);
            Console.WriteLine("init bloom filter ok.");
            //JustMathCore.RocksDB.DB db = new JustMathCore.RocksDB.DB();
            //db.Open(dbpath, false);
            var logfile = System.IO.Path.Combine(dbpath, "log_" + DateTime.Now.ToFileTimeUtc() + ".txt");
            string tag = "";

            foreach (var f in files)
            {
                string[] addrs = System.IO.File.ReadAllLines(f);
                foreach (var add in addrs)
                {
                    JustMathCore.Hex160 hex;
                    if (add.Length == 40 || add.Length == 42)
                    {
                        continue;
                        //hex = JustMathCore.Hex160.FromString(add);
                    }
                    else if (add.Length == 43)
                    {
                        var addr = add.Substring(9);
                        hex = Hex160.FromBytes(AddrTools.base58Decode(addr).Skip(1).Take(20).ToArray());
                        tag = add.Substring(0, 8);
                    }
                    else
                    {
                        continue;
                        //throw new Exception("error format.");
                    }

                    var result = bt160.Exist(hex);
                    if (result == true)
                    {
                        var line = "find:" + hex.Format() + " " + add + " tag=" + tag + "\n";
                        Console.WriteLine(line);
                        System.IO.File.AppendAllText(logfile, line);

                    }
                }
                Console.WriteLine("done " + f + DateTime.Now.ToLongTimeString());

            }

            //db.Dispose();

        }
        //从原始地址数据，直接入库
        public static void DoImport2DB(string type, string[] srcfiles, string dbpath)
        {
            JustMathCore.AddrTools.AddrType addrtype;
            if (!Enum.TryParse<JustMathCore.AddrTools.AddrType>(type, true, out addrtype))
            {
                Console.WriteLine("error type:" + type);
                return;
            }
            var txtdat = System.Text.Encoding.UTF8.GetBytes(type);
            if (txtdat.Length > 8) txtdat = txtdat.Take(8).ToArray();
            //LinePool curpool = new LinePool(outdir, 32);
            BitArray[] bitarrays = new BitArray[5];
            for (var i = 0; i < 5; i++)
            {
                bitarrays[i] = BitArray.Create();
            }

            //ThreadWriteMgr.Start(curpool, 16);
            JustMathCore.RocksDB.DB db = new JustMathCore.RocksDB.DB();
            db.Open(dbpath, true);
            var wb = db.CreateWriteBatch();

            string[] src = new string[8];
            foreach (var f in srcfiles)
            {

                DateTime t0 = DateTime.Now;
                var finfo = new System.IO.FileInfo(f);
                Console.WriteLine("begin " + f + " at:" + t0.ToLongTimeString());
                JustMathCore.AddrTools.AddrType addrtypesub = addrtype;
                //NEO有子类型
                if (addrtype == AddrTools.AddrType.NEO)
                {
                    if (f.Contains("ont"))
                    {
                        addrtypesub = AddrTools.AddrType.ONT;
                    }
                }
                if (addrtype == AddrTools.AddrType.BTC)
                {
                    if (f.Contains("bch"))
                    {
                        addrtypesub = AddrTools.AddrType.BCH;
                    }
                }
                if (finfo.Length > 1 * 1024 * 1024 * 1024)
                {
                    Console.WriteLine("file is too large,use stream mode.");
                    using (var fs = finfo.OpenRead())
                    {
                        using (var sr = new System.IO.StreamReader(fs))
                        {
                            string[] lines = new string[16 * 1024 * 1024];
                            int linecount = 0;
                            while (true)
                            {
                                var line = sr.ReadLine();
                                if (line == null)
                                    break;
                                lines[linecount] = line;
                                linecount++;
                                if (linecount == lines.Length)
                                {
                                    Console.WriteLine("stream mode step.");

                                    PForLinesDB(db, addrtypesub, txtdat, bitarrays, lines, lines.Length);
                                    linecount = 0;
                                }
                            }
                            if (linecount > 0)
                            {
                                PForLinesDB(db, addrtypesub, txtdat, bitarrays, lines, linecount);
                            }
                            lines = null;
                        }
                    }
                }
                else
                {
                    var lines = System.IO.File.ReadAllLines(f);
                    PForLinesDB(db, addrtype, txtdat, bitarrays, lines, lines.Length);
                }

                var time = (DateTime.Now - t0).TotalSeconds;
                Console.WriteLine("=>end " + f + " " + time);
                //ThreadWriteMgr.WaitWrite();
            }
            for (var i = 0; i < bitarrays.Length; i++)
            {
                System.IO.File.WriteAllBytes(System.IO.Path.Combine(dbpath, "bloom4G_" + i + ".fbin"), bitarrays[i].Memory);
            }
            //);
            //ThreadWriteMgr.WaitToExit();
            wb.Dispose();
            db.Dispose();
            //curpool.Dispose();
        }
        private static void PForLinesDB(JustMathCore.RocksDB.DB db, JustMathCore.AddrTools.AddrType type, byte[] txtdat, BitArray[] bitarrays, string[] lines, int count)
        {
            int threadcount = 8;
            RocksDbSharp.WriteBatch[] wbs = new RocksDbSharp.WriteBatch[threadcount];

            var conv = AddrTools.GetConv(type);
            System.Threading.Tasks.Parallel.For(0, threadcount, (seed) =>
            {
                int first = 0;
                var wb = db.CreateWriteBatch();
                wbs[seed] = wb;
                for (var l = seed; l < count; l += 8)
                {
                    var line = lines[l];
                    {
                        if (line.Length < 34)
                            continue;
                        byte[] data = null;
                        try
                        {
                            data = conv.Addr2Hash(line);
                            if (first == 0)
                            {
                                var addrr = conv.Hash2Addr(data);
                                Console.WriteLine("addr :" + addrr + " <= " + line);
                                first++;
                            }
                        }
                        catch
                        {
                            continue;
                        }
                        //if (line.Length == 40)
                        //{
                        //    for (var i = 0; i < 20; i++)
                        //    {
                        //        var d = byte.Parse(line.Substring(i * 2, 2), System.Globalization.NumberStyles.HexNumber);
                        //        data[i] = d;
                        //    }
                        //}
                        //else if (line.Length == 34)
                        //{
                        //    var b58 = AddrTools.base58Decode(line);
                        //    for (var i = 0; i < 20; i++)
                        //    {
                        //        data[i] = b58[i + 1];
                        //    }
                        //}
                        //else
                        //{
                        //    //如果遇到不规范的忽略他，只支持两种文本导入格式
                        //}

                        //for (var i = 0; i < txtdat.Length; i++)
                        //{
                        //    data[24 + i] = txtdat[i];
                        //}
                        lock (bitarrays)
                        {
                            var index = BitConverter.ToUInt32(data, 0);
                            bitarrays[0][index] = true;
                            var index1 = BitConverter.ToUInt32(data, 4);
                            bitarrays[1][index1] = true;
                            var index2 = BitConverter.ToUInt32(data, 8);
                            bitarrays[2][index2] = true;
                            var index3 = BitConverter.ToUInt32(data, 12);
                            bitarrays[3][index3] = true;
                            var index4 = BitConverter.ToUInt32(data, 16);
                            bitarrays[4][index4] = true;
                        }
                        wb.Put(data, txtdat);
                        // ThreadWriteMgr.Queue(data);
                    }
                }
            }
              );
            for (var i = 0; i < threadcount; i++)
            {
                if (wbs[i] != null)
                {
                    db.Write(wbs[i]);
                    wbs[i].Dispose();
                }
            }
            wbs = null;
        }

        //从原始地址数据，只处理BloomFilter
        public static void DoImport(string type, string[] srcfiles, string dbpath)
        {
            var txtdat = System.Text.Encoding.UTF8.GetBytes(type);
            if (txtdat.Length > 8) txtdat = txtdat.Take(8).ToArray();
            //LinePool curpool = new LinePool(outdir, 32);
            BitArray[] bitarrays = new BitArray[5];
            for (var i = 0; i < 5; i++)
            {
                bitarrays[i] = BitArray.Create();
            }

            if (System.IO.Directory.Exists(dbpath) == false)
                System.IO.Directory.CreateDirectory(dbpath);
            //ThreadWriteMgr.Start(curpool, 16);
            //JustMathCore.RocksDB.DB db = new JustMathCore.RocksDB.DB();
            //db.Open(dbpath, true);
            //var wb = db.CreateWriteBatch();

            string[] src = new string[8];
            foreach (var f in srcfiles)
            {

                DateTime t0 = DateTime.Now;
                var finfo = new System.IO.FileInfo(f);
                Console.WriteLine("begin " + f + " at:" + t0.ToLongTimeString());

                if (finfo.Length > 1 * 1024 * 1024 * 1024)
                {
                    Console.WriteLine("file is too large,use stream mode.");
                    using (var fs = finfo.OpenRead())
                    {
                        using (var sr = new System.IO.StreamReader(fs))
                        {
                            string[] lines = new string[16 * 1024 * 1024];
                            int linecount = 0;
                            while (true)
                            {
                                var line = sr.ReadLine();
                                if (line == null)
                                    break;
                                lines[linecount] = line;
                                linecount++;
                                if (linecount == lines.Length)
                                {
                                    Console.WriteLine("stream mode step.");

                                    PForLines(txtdat, bitarrays, lines, lines.Length);
                                    linecount = 0;
                                }
                            }
                            if (linecount > 0)
                            {
                                PForLines(txtdat, bitarrays, lines, linecount);
                            }
                            lines = null;
                        }
                    }
                }
                else
                {
                    var lines = System.IO.File.ReadAllLines(f);
                    PForLines(txtdat, bitarrays, lines, lines.Length);
                }

                var time = (DateTime.Now - t0).TotalSeconds;
                Console.WriteLine("=>end " + f + " " + time);
                //ThreadWriteMgr.WaitWrite();
            }
            for (var i = 0; i < bitarrays.Length; i++)
            {
                System.IO.File.WriteAllBytes(System.IO.Path.Combine(dbpath, "bloom4G_" + i + ".fbin"), bitarrays[i].Memory);
            }
            //);
            //ThreadWriteMgr.WaitToExit();
            //wb.Dispose();
            //db.Dispose();
            //curpool.Dispose();
        }
        private static void PForLines(byte[] txtdat, BitArray[] bitarrays, string[] lines, int count)
        {
            int threadcount = 8;
            //RocksDbSharp.WriteBatch[] wbs = new RocksDbSharp.WriteBatch[threadcount];

            System.Threading.Tasks.Parallel.For(0, threadcount, (seed) =>
            {
                //var wb = db.CreateWriteBatch();
                //wbs[seed] = wb;
                for (var l = seed; l < count; l += 8)
                {
                    var line = lines[l];
                    {
                        var data = new byte[20];
                        if (line.Length == 40)
                        {
                            for (var i = 0; i < 20; i++)
                            {
                                var d = byte.Parse(line.Substring(i * 2, 2), System.Globalization.NumberStyles.HexNumber);
                                data[i] = d;
                            }
                        }
                        else if (line.Length == 34)
                        {
                            var b58 = AddrTools.base58Decode(line);
                            for (var i = 0; i < 20; i++)
                            {
                                data[i] = b58[i + 1];
                            }
                        }
                        else
                        {
                            //如果遇到不规范的忽略他，只支持两种文本导入格式
                        }

                        //for (var i = 0; i < txtdat.Length; i++)
                        //{
                        //    data[24 + i] = txtdat[i];
                        //}
                        lock (bitarrays)
                        {
                            var index = BitConverter.ToUInt32(data, 0);
                            bitarrays[0][index] = true;
                            var index1 = BitConverter.ToUInt32(data, 4);
                            bitarrays[1][index1] = true;
                            var index2 = BitConverter.ToUInt32(data, 8);
                            bitarrays[2][index2] = true;
                            var index3 = BitConverter.ToUInt32(data, 12);
                            bitarrays[3][index3] = true;
                            var index4 = BitConverter.ToUInt32(data, 16);
                            bitarrays[4][index4] = true;
                        }
                        //wb.Put(data, txtdat);
                        // ThreadWriteMgr.Queue(data);
                    }
                }
            }
              );
            //for (var i = 0; i < threadcount; i++)
            //{
            //    if (wbs[i] != null)
            //    {
            //        db.Write(wbs[i]);
            //        wbs[i].Dispose();
            //    }
            //}
            //wbs = null;
        }
    }
}
